{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Cross-Entropy Method\n",
    "\n",
    "---\n",
    "\n",
    "In this notebook, we will train the Cross-Entropy Method with OpenAI Gym's MountainCarContinuous environment."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1. Import the Necessary Packages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import gym\n",
    "import math\n",
    "import numpy as np\n",
    "from collections import deque\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "from torch.autograd import Variable"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2. Instantiate the Environment and Agent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[33mWARN: gym.spaces.Box autodetected dtype as <class 'numpy.float32'>. Please provide explicit dtype.\u001b[0m\n",
      "\u001b[33mWARN: gym.spaces.Box autodetected dtype as <class 'numpy.float32'>. Please provide explicit dtype.\u001b[0m\n",
      "observation space: Box(2,)\n",
      "action space: Box(1,)\n",
      "  - low: [-1.]\n",
      "  - high: [1.]\n"
     ]
    }
   ],
   "source": [
    "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
    "\n",
    "env = gym.make('MountainCarContinuous-v0')\n",
    "env.seed(101)\n",
    "np.random.seed(101)\n",
    "\n",
    "print('observation space:', env.observation_space)\n",
    "print('action space:', env.action_space)\n",
    "print('  - low:', env.action_space.low)\n",
    "print('  - high:', env.action_space.high)\n",
    "\n",
    "class Agent(nn.Module):\n",
    "    def __init__(self, env, h_size=16):\n",
    "        super(Agent, self).__init__()\n",
    "        self.env = env\n",
    "        # state, hidden layer, action sizes\n",
    "        self.s_size = env.observation_space.shape[0]\n",
    "        self.h_size = h_size\n",
    "        self.a_size = env.action_space.shape[0]\n",
    "        # define layers\n",
    "        self.fc1 = nn.Linear(self.s_size, self.h_size)\n",
    "        self.fc2 = nn.Linear(self.h_size, self.a_size)\n",
    "        \n",
    "    def set_weights(self, weights):\n",
    "        s_size = self.s_size\n",
    "        h_size = self.h_size\n",
    "        a_size = self.a_size\n",
    "        # separate the weights for each layer\n",
    "        fc1_end = (s_size*h_size)+h_size\n",
    "        fc1_W = torch.from_numpy(weights[:s_size*h_size].reshape(s_size, h_size))\n",
    "        fc1_b = torch.from_numpy(weights[s_size*h_size:fc1_end])\n",
    "        fc2_W = torch.from_numpy(weights[fc1_end:fc1_end+(h_size*a_size)].reshape(h_size, a_size))\n",
    "        fc2_b = torch.from_numpy(weights[fc1_end+(h_size*a_size):])\n",
    "        # set the weights for each layer\n",
    "        self.fc1.weight.data.copy_(fc1_W.view_as(self.fc1.weight.data))\n",
    "        self.fc1.bias.data.copy_(fc1_b.view_as(self.fc1.bias.data))\n",
    "        self.fc2.weight.data.copy_(fc2_W.view_as(self.fc2.weight.data))\n",
    "        self.fc2.bias.data.copy_(fc2_b.view_as(self.fc2.bias.data))\n",
    "    \n",
    "    def get_weights_dim(self):\n",
    "        return (self.s_size+1)*self.h_size + (self.h_size+1)*self.a_size\n",
    "        \n",
    "    def forward(self, x):\n",
    "        x = F.relu(self.fc1(x))\n",
    "        x = F.tanh(self.fc2(x))\n",
    "        return x.cpu().data\n",
    "        \n",
    "    def evaluate(self, weights, gamma=1.0, max_t=5000):\n",
    "        self.set_weights(weights)\n",
    "        episode_return = 0.0\n",
    "        state = self.env.reset()\n",
    "        for t in range(max_t):\n",
    "            state = torch.from_numpy(state).float().to(device)\n",
    "            action = self.forward(state)\n",
    "            state, reward, done, _ = self.env.step(action)\n",
    "            episode_return += reward * math.pow(gamma, t)\n",
    "            if done:\n",
    "                break\n",
    "        return episode_return\n",
    "    \n",
    "agent = Agent(env).to(device)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3. Train the Agent with the Cross-Entropy Method\n",
    "\n",
    "Run the code cell below to train the agent from scratch.  Alternatively, you can skip to the next code cell to load the pre-trained weights from file."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 10\tAverage Score: -1.44\n",
      "Episode 20\tAverage Score: -3.98\n",
      "Episode 30\tAverage Score: -4.18\n",
      "Episode 40\tAverage Score: 2.57\n",
      "Episode 50\tAverage Score: 18.74\n",
      "Episode 60\tAverage Score: 29.35\n",
      "Episode 70\tAverage Score: 38.69\n",
      "Episode 80\tAverage Score: 45.65\n",
      "Episode 90\tAverage Score: 47.98\n",
      "Episode 100\tAverage Score: 52.56\n",
      "Episode 110\tAverage Score: 62.09\n",
      "Episode 120\tAverage Score: 72.28\n",
      "Episode 130\tAverage Score: 82.21\n",
      "Episode 140\tAverage Score: 89.48\n",
      "\n",
      "Environment solved in 47 iterations!\tAverage Score: 90.83\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYoAAAEKCAYAAAAMzhLIAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJztnXl4W+WV/z9Hkh3bSbxlceI4+56wZDEhhKVAKDuEUqZAKVBKmy600E4plNJfO8xMO9NpgZalhZTSAtOyQ6GFMkCAskMSyEZCEmd3Vifxks2OJb2/P+69sixL1mLJkq/P53n8WLr36ur4Wnq/9yzvecUYg6IoiqLEwpNtAxRFUZTcRoVCURRF6RQVCkVRFKVTVCgURVGUTlGhUBRFUTpFhUJRFEXpFBUKRVEUpVNUKBRFUZROUaFQFEVROsWXbQPSwcCBA82oUaOybYaiKEqPYsmSJXuMMYPiHecKoRg1ahSLFy/OthmKoig9ChHZnMhxGnpSFEVROkWFQlEURemUjAuFiDwoIrtFZGXYtnIReUVE1tm/y+ztIiJ3iUiNiCwXkRmZtk9RFEXpnO7wKP4EnB2x7YfAQmPMeGCh/RzgHGC8/TMf+F032KcoiqJ0QsaFwhjzJrAvYvM84CH78UPARWHbHzYW7wOlIjI00zYqiqIosclWjqLCGLPDfrwTqLAfDwO2hh1Xa29TFEVRskTWk9nGWmIv6WX2RGS+iCwWkcV1dXUZsExRFEWB7AnFLiekZP/ebW/fBgwPO67K3tYBY8wCY0y1MaZ60KC480UURVF6BDsbm3n0wy34A8FsmxIiWxPungeuBv7b/v1c2PZvi8hjwPFAY1iISulhbN13CK9HqCwtTPq12xoOs6K2gQ17DnLJzCoG9y/IgIWKkn0eX7SFd2r28q3TxgLw5QcXsbOpmfc37OWOL0yj/tARnl5Sy96DR2huDTBxSH+OH13OuMH9u83GjAuFiDwKnAoMFJFa4KdYAvGEiFwLbAa+YB/+InAuUAMcAq7JtH1KZmjxB7j0/vcoLszjHzecjIgA8PGWeu54ZS0HWvw8+60TO7xuf3Mrv3jpU/73/S2hbSu3NfLbK2Z2m+2RvL1uDwAnjR+YNRvcijGG2vrDDC8vAqA1EOSrDy3m0uOGc+7RVh3LEX+QPK+EPkPxCAYNTc2tlBblxzzm0BE/hXnehM8ZiyP+IAtX7+L0yYPp4/Mm9VpjDL96eQ33vr4er0f42/LtFPi8FBf6+PKcUfzp3U3sbGxm5bZGDh4J0MfnIc/r4UCLH4DvnTGBG84Y3yX7EyXjQmGMuTzGrrlRjjXAdZm1SOkOnlhcy/bGZrY3NvPRlnpmjizn7oXruP2VtQB4wr6f+w4e4d7Xa6itP8TSrQ3s3t/CV04czbxplby6ehd3v1bD+xv2Mn5wP773xDLGDOzLjWdNpF+fjh/fQNCwcc8BGg/7KS3KY+ygfl36O5bXNvCVPy2iT56Hd394Ov0L8njgrQ08v2w7/33xMUypLO709fubW+nXx9flAcmNvLFmN79ZuI6PtzRw9+XTueDYSt5aV8c/19axdGsDs0aXI8Dnf/cuRfk+7vnidMYM6kdzawCAgrz2A/OupmbuWriOV1fvYldTC6dNHMS3ThvHzBFleMI+cA2HjnD67f9kztgB/Oay6Xg9if9vPtpSz4J/buBnnzuKAf36cNfCddzzeg1fPWk0Pz5/Srtjl2yu5+/Lt/Phxn2cMGYAN58ziTyvFe3f0XiYnz73CS+v2sXls4Zz45kTWfDWBlZtb+K/P38Mw0oLKS7M466F6zh76hB+cPZExg7qhzGGzXsP8auX1/DrhWs5blQZc8Zl/gZGrLG5Z1NdXW2011Pu0OIPcOov32Bw/z5sqDvIGVMq+N4ZEzjjjn9y+qTBDC8v5PdvbWTjf52LiPDc0m3c8NhSRg/sy+iBffn26eOYMaIMgObWAHNv/yf9+vgI2l+S1mCQocUF3HflTI6pKm333rf97RP++M6m0POvnjSaG8+a2GFQiUVrIMiPnllBvwIfl88awbUPLeJgS4B9B4/ww3MmMW9aJaf+8g1a/EHyfR4uP244a3btZ9OeQ0ytLGbmqDI+O7mCoaWF3P7yGv707ibmTqrgPy6aytCSjiG4PQdaeHzRVp7+qJbSwjz+55JjuhRSMMZw6YL3+cyEQVx32rhOj9205yCLNu3jjMkVlPWNfffdFZZtbWDNzv38S3VVO7H88webufXZlQwrLcQfDFJVVsTT35zDt//yEf9cW0dLa5DPTqmgbn8LS2sbKMr30uoPMn1EGYs27aOqrJC/feckivKtm4UWf4Av3Pcen+7cz+mTBjNyQF8eX7SF+kOtFBf4OG5UObfNm0pVWREL3lzPz1/8FIArZ4/k3+dNbXfHHotA0HDeXW/x6c79zBk7gB+dO5mL7n2Hvn18NDW38uTXT6B6VDkA63bt5+zfvIXPI0weWszSrQ2cOG4AXzp+JIs31/Poh1sIBA03njmRr548OuaNRMOhI1E9o0NH/Jx/99vsb/bzjxtOZmC/Pin9f0RkiTGmOt5xrmgKqOQWTyyuZUdjM/9zyTG8umoXj364lT0HWvB6hNvmTeXxRVYFdCBo8HmF1oB1s/LQNbMYMaCo3bkK8rzccu4kvv2Xj+mb7+Whr8wi3+fhmj9+yANvbeSuy6eHjg0GDS8s38EJYwYw/zNjeG31bh54eyPvbdjLM9+aQx+fl+0Nh7nhsY+54viRzJtW2eEL+u9/W8WTS2rxeoQ/vrMJn0d4/Osn8OtX1/LAWxv5ZHsTxsBz153I7a+s5aH3NjNlaDGzRpfzyfZGFn66m/95aQ0FeR5a/EHOmFzBW+vq+Owdb3LvFTP4zIT2hRdXP/ghn2xvYtbocmp2H+D8u9/mPy86mktmVqV07dfuOsCHG/fx4cZ9DC0p4OIZ1nmO+IPc+uwK9hxo4bYLj6L+0BG+/McPqT/Uis8jnHfMUG7/l2Px2QPlzsZmhpQUhP5Pv3l1LTsam2nxB2nxB/AHDFeeMJJTJw6Oakdza4Bfv7qOBW+uJ2jg0537+X/nT0ZE+GR7I7f9bRWfmTCI319VzcPvbeI/X1jNok37eGXVLi49bjjlffP59avrAPjNZdOYNbqcm59ewY6Gw8ybVsmTS2r5+Yur+c+LjgbgZy+sZlltI/d9aSZnHzUEgOvnjuMfK3ayeHM9zy3dxi3PrOBP18zi4fc2M2t0OdOHl3L/mxt4+qNaDh0JUJTvZdbocuaMHcCcsQPxiPB/n+wE4JunjuX5pdv5dOd+Lji2kr8t284X7n+PksI8nvv2iVx6//v84KnlvHj9yRTme/nZi6spyvfy+o2nMrBfH55aUsuPnlnBOzV7yfd6OG3SIH583pRQyC0WscJnRfk+7rl8Bhf99h3ueGUtP//c0Ul8SpJHhUJJir0HWvjC/e9x9+UzYoZd/vj2RqaPKOWkcQOpKC7gofc289a6PXzvjAlUFBeEXP2AMfiAQNCq7vB6o99VnXf0UHad38Lxo8s5algJABXFBbRGVIUsrbXCVj86dzKnTRzMaRMHM6WymFueWcFHmxs4YewA/rFyJ4s21bNoUz0vrNjB98+cwKQhxQSDhkfe38wj72/m66eM4UuzR/LHdzZx1LBiZo4s41unjuPy37/P35ZtZ/4pYzh2eCkPf2UWza2Bdt7KrqZmXv5kJ0u3NvLF44czc2Q5W/YeYv4ji/n2Xz7iuetOZIwdDtu45yCfbG/ix+dN5qsnj2F3UzPffXwpNz21jOFlhRw/ZkDS/593aqx8ytHDSrj56eW0BoKcOnEwtz67gldX76Ywz8tZv34Tj0B5v3zuvHQaC1fv5pH3NzNn7AAuPW4ETy2p5cYnl/HMt+YwY0QZ72/Yy12v1TCwXx/69fFSkOel/tARvvm/H/Hct09kQkVHD+jf/76Kv3ywhUurh9Mnz8OD72xkf3Mr00aU8oe3NlJWlMcdXziWfJ+HS2ZW8cv/W8N3/vIxLf4gn5s+jCmV1l347DEDmDfNmkr18Fdmhc5fXJDHA29vZPTAftTsPsCjH27hayePDokEWIPp52dW8fmZVUyo6Mdtf1vFD55aRm39YW49dzJnHzWEgf36sK3hMENKCthWf5h31+/h52vayu1FwBh4dfUu9hxoYfqIUu66bBrlRXk89N5m/uvio6kqK+KXlxzDFX/4gMsWvMcVs0fyxpo6bj13cuhO/5KZVVSPLGPvwRamVpYk7OF2xpTKYh68+jimjSiNf3AXUaFQkmJr/WHW1x3kicVb+bcLp3bYv6upmQ17DvLj86y7xwkV/Zk9ppzNew8x/5QxAG1CETT2b+u13hjut4hw7Umj223zeiT0eoeXP9mFzyOcFnaXe/4xQ/nxX1fy3vo9nDB2AO/W7GHkgCK+dPxIfvXyGl5ZtYtJQ/qze38L+w4e4TMTBnHT2ZPweoSfXNAWc549ppwZI0rZsOcg153aFtKJ/MJXFBdw5QmjuPKEtm0jBhTx+6uqufCet5n/yBKe/dYc+hfk8coq627VGdwGFxew4KpqLrj7bW54bCkv3nAy5REhoZ2NzfzcvlutLC3kayePoTC/zYZ311t/3/9eezyXLniPm59eEdr3H/OmMndyBT957hPq9jez4KpqKooL+MyEQSyvbeCuhTWcPqmC/3pxNQDPL93OjBFlvPzJTgryPLx102mh99rV1Mx5d73NNx5Zwq8vm8bGPQeZNKSYiUP6c7DFz18/3sa/zKziF5ccgzEGn8cSiyeX1JLv8/DwV2YxwB5ES4vyOf+YSp7+qJbRA/sybXgpIsKfrplFLG48ayJvrqvjP/6+ioI8D/8ys4qbzp4U8/grZ4/kqSW1PPPRNipLCvjslApEhK/Zn8lwdjU18976vRwJBDl90mCW1zbw3ceW0tTs594vzkBE+MkFU7li9siQSM4ZN5D7vjSTf318KTc9tZyRA4q4as7IducdNbAvowb2jWljKnRXgYUKhZIUzt3/Syt38pPzp7RLEgIs2mR1aznOjtUC/O6KmbQGgqFBxhGEkFDYeTJPErN6ogrFqp3MHjOAkqK80Lb+BXkcPayEd9bv5fpAkA827uPCaZV87ZQxfH5mFc9+vI2XVu5gytBiTpkwiLOPGhI1uSki3HflTA61BNqdP1GGlxdx7xdncOWDH/LzFz/lvy4+mldW7WLK0GKqytrCD/36+Lj78ulc/Nt3+dEzK7jvyvbVXve/uZ4XVuygtDCPvQePcOzw0lA4yx8I8sGGfZx/bCUlRXm8cP3JrNjWyJtr65haWczcyVYDhAeubh+SFhG+f+ZErnrwQy65713qDx1h8tBiXlyxg/93/hReXrWLU8YPaidIFcUF3PvF6XzxgQ+48J53AKgsKeC1G0/lHyt3cuhIgEuPGx46/08umMINc8fT7A9QmO+luKD9NbzyhJE8/VEtF08fllDivyDPy5+umcWq7U3MGTcglKuIhc/r4WefO5qLf/sOV80ZFQqxRaOiuICLprc1hDh9UgUvXH8ya3ftD+UgvB7p4EmdNXUIz153Irf97RO+deq4pKugchkVCiUpnLv/nU3NLKttYLqddHZYtHEfhXnedmGpyESpMxDbmkPAPqkvCaXweiQkMAA1uw+woe4gX54zqsOxJ44bwP3/3MD7G/ZxoMXPnLFWSKe8bz7XnjS6g7cSi8H9C6ALpetzxg3kytkjefi9TVx4bCVLNtfzndM7ljceNayES6qreO7j9nNN9ze38uTiWi44ZijXnDiaefe+ExJugBXbGtnf4ufEcdbf5/UI04aXMm14/NDEyeMHMmt0OR9u3MdVJ4xk5sgybnhsKX98ZyM7Gpu58cyJHV5z/JgBPPmNE9i67xCBoOFfn1jGQ+9u4vU1uxk1oIiZI9t/NkqK8ighushOG17KY/NnJ2SrQ2VpYVJzdKYNL+XNm06LWlQQj+HlRXHzCQATKvrz56/OTvr8uU7WW3goPQt/2MD00sqdHfYv2lTP9BGlnVaPOELhnMvOZccMPUXDI+09ipftMM5np1R0OHbO2IH4g4bfLLRKc2enEPtPFzfMHU/fPj6+/shigia6vWB5FoGIisSnl9RyoMXPl08cHRa+a9v/7vq9AJyQwt8nIvzbBVO58NhKvv/ZicydXEEfn4fbX16L1yPMnRw9aT1jRBnzpg3j4hlVnDZxkF3KvI9LZlYlXRI8e8yAtMTuO6OqrCipcljFQoVCSQpHJ0oK8/jHyp2El1c3Nbfy6c6mdmGnaHjCktnWOVMLPQXD3nv51kbGDOwb9W5x5sgy8n0eFm2qZ9KQ/imXEqaDsr75fOf0cTQ1+xlWWsjUGAUBHhHCNJlg0PDQe5uZPqLUjuFb28PF8t31e5g0pH8o9p8sUyqLuevy6ZQU5dGvj49TJw7icGuA40eXdzp5zeHmcyZx8IgfEfjcjNSqtpTcRIVCSQrHCzhragVb9h1i1Y6m0L6PNtcTNMQVCl9E6MlvD3bJhp78gbZB0h8MxrwbLcjzMtMOkZ0wNnvehMPVc0YxsaI/F8+IHY/3RYTW3lm/h4172kJrofCdfUxrIMjiTfXMGZu+5KYzM/rMGF5PJJOGFPP1U8Zy2XHDGZZC2xYld9EchZIUzsB0+qQKnlhcy5LN9UyttEpWF2+qx+sRpscp13NCTI7oBFNJZou0C4MFgqbTkMKJ4wbw3oa9aR1IU6WPz8tL3z2509CMx07WG2MQEbbuOwzA8aPt/ENEQcChIwFa/EEqS9PXE+uco4ZSd14L/1I9PP7BNj88J3blkdJzUaFQksK5i68sLcDnEXY2Nof2fbhpH1Mri+kbpbVGOJ7IZHaKHkWLP9yj6FwoLpk5nF1NLZycI/2a4sXvQ16XAa+0hemcv9ET4VE44bt0xt/zfR6+enLH8lGl96GhJyUpnIEpz+thcP8+7GyyhMIYw4raxlDrjc5w8tzO4OeEnpIZ4zwR5bFB07lQDCkp4D8uOirjydJ00WGuSagyzNrueBTO/yNUYqw9pZQMoEKhJIU/7M61oqSAXbZQ1B9q5XBrgBEJlBB6bc/BKe0MBg0eiX+XHU5kDN8f6FwoehqeDnNN7O2ORyHtq57aCgLccw2U3EGFQkmKQJhQDCkuCIWetjdYMfRhZfGTmN6IQc4fNEmFncApj217HjQmqfLaXMcXURkWanMSCj1ZxzkC4ThXbroGSu6gQqEkRUgoRKgoLmBXUwsAtfW2UCRQ7RIKPYUGOZNUIts5RzAs9BQvmd3T8ESGnmxRDIWeIoUklMPoTiuV3oJ+rJSkaOdRlBRwoMXPgRZ/yKNIZKZsW+jJhH4neyfs83iSqnrqaTj9EduukfW3OiGnyBxFKPSkHoWSAVQolKQIF4qKYmti187GZrY3HKYgz0NZAn2QIpPZqQzyHo8Q5lAQiJPM7ml4vZFiam33RVY9hYktpLfqSVEcVCiUpHAGd5/HCj2B1W1ze+NhhpUWJpSQ7pCoTUEovNJ+VrLbktkxq5o6JLMjQ0/uuQZK7pBVoRCR74nIJyKyUkQeFZECERktIh+ISI2IPC4imVl6S0mJQFh1zRBbKHY2NrOt/nDCDdqcxHX4IJiKR9GhPNZFYRfH6/KHhZ58YdcoVBBgXwINPSmZJGtCISLDgOuBamPMUYAXuAz4BXCnMWYcUA9cmy0blY60TY6T0ApoO5ua2dbQnHDbBidx7UzeC6TgDfgihMJ1OQpHTMNCT+Glr5FVTzqPQskk2Q49+YBCEfEBRcAO4HTgKXv/Q8BFWbJNiUK4R1GU76N/gY+t+w6x50BLwh5FtLBKst5AZJtx9wmF9Tvcowi/RpG9npy8vlY9KZkgax8rY8w24FfAFiyBaASWAA3GGL99WC0wLPoZlGwQ7lEADCkuYOnWBiCx0lgAn7d9fD0YNElPFLO6q4YJhcuS2R3zOLQLPYX2O0KhHoWSQbIZeioD5gGjgUqgL3B2Eq+fLyKLRWRxXV1d/BcoacEfUV0zpKSANbv2A4mVxkLHQdCacJd86MkfLhQuS2Z3yOMEg+1DT45XplVPSjeQTUf1DGCjMabOGNMKPAOcCJTaoSiAKmBbtBcbYxYYY6qNMdWDBg3qHouVdhPuwFo20okAJepRdOhjZFLwKDxRPAoX3U17I/M4pr2YRi5cFFkVpSjpJJtCsQWYLSJFYtVUzgVWAa8Dl9jHXA08lyX7lChE3rk6lU8ihJLb8YicVRxMwaPwSmSOwl2DpCcyjxOZzHYm5EV2j3WRWCq5QzZzFB9gJa0/AlbYtiwAbgb+VURqgAHAH7Jlo9KRQEQDvwpbHAb370O+L7GPU6RH4Q+apGPr3g5VT8GkxSaXiczjRP59IoJHCK0wqKEnJZNkdT0KY8xPgZ9GbN4AzMqCOUoCWCGQNkFwPIpkFrmPXHQnmMqEO5eXxzrC6Q8vj40Q0/Br4HgW6lAomUCL6ZSkCATbN/BLSSgiSjtTSWa7vzw2MvQUDHkZDhIWfguVx6pSKBlAhUJJikBES/CKEqvfU1UKQuEkaoOpJLNFMCYs9OKy8tiOCf+OIuANKxF2BMVN10DJHVQolKRwchQOA/v24fMzqjhzakXC54icA5Ba99jI8JW75hBEhuciy2PBCT1Zj7XqSckkuma2khSBoMEXNv3X4xFu/8KxSZ3DCaEEw5LZqfR6cl7r84Lf9cnsjuE5j0RZM9tFYqnkDupRKEmRSoVSJN6IRG2qyWywBkpjDEHjrrvpaF5XtGR2UKuelG5AhUJJilTmPETiiUzUppBfCA/NRLYVcQOhHEUgzKPwRnoUbVVP2sJDySQqFEpSpBImiiQyv5DSehRh53DjWgwdlzrtKAKedh5F+9cpSjpRoVCSIpiG6qKO60Gn1j3Wea1TGuqmu+lQaC3GhDuwZ6d3aDPejUYqvQYVCiUp0uFRdKzoSa3XE1gDpLN2tqtCTx0m3HW8Rt6w5WCNVj0pGUSFQkmKVBLPkXQIq6TYPdayp22ymZsGyY4T7jp6XR5PlO6xLvKqlNxBhUJJCn/EAjqp0CFRm8KEu7Y77qA7PYoo4bmoyWytelK6ARUKJSkCwa4PRt6I0s9UKqk8YR6FGyebdez1FKU8NlrVk4uugZI7qFAoSREIBtOWzG434S7pZLZtjzGuDLtETkqMXI8CYlQ9uegaKLmDCoWSFAGTnvCGzyPtPIqkQ092vynXzqOI1j02StVTsEMLj+6zUek96MdKSYp0eBRgDXr+MI8ilYWLLHuMK5PZHSYlRimP9USILbirRFjJHVQolKRIVzvvyM6nyXsUbfa4MZkdbVJi5DXyCB27x6pQKBlAhUJJilQmx0XDF975NKUchfXRDRrjykRuIpMSw9fkcI5z0zVQcgcVCiUpopVppoLHIwRsTyCVSXyOR+EPmlAIy0130x0mJUZLZkepetLyWCUTZFUoRKRURJ4SkU9FZLWInCAi5SLyioiss3+XZdNGpT3RyjRTwRsRX0+6zXiUpoBuGiQ7TEoMRJ+Z7Szyp1VPSibJtkfxG+AlY8wk4FhgNfBDYKExZjyw0H6u5AjR7mxTIXLRna60GQ+6sCFetEmJnfV6CmrVk5JBsvaxEpES4BTgDwDGmCPGmAZgHvCQfdhDwEXZsVCJhj/KnW0qWIOcNcJ3tXusG5PZkZMSoyWzReg4M1s9CiUDZPP+YzRQB/xRRD4WkQdEpC9QYYzZYR+zE0h8jU0l4wQz4VGkkswOL491aTJbpH0vpw4ehUc69npy0TVQcodsCoUPmAH8zhgzHThIRJjJWC0xTbQXi8h8EVksIovr6uoybqxikY7usdC2OpuzOl2XPIqAO++mvSKdt/AIz/PYv8Vl10DJDbIpFLVArTHmA/v5U1jCsUtEhgLYv3dHe7ExZoExptoYUz1o0KBuMVhJT/dYsAY5fxcS0eFtxt24cBF0LH+N/Ps80tZmPB3rhChKLLImFMaYncBWEZlob5oLrAKeB662t10NPJcF85QYpMujcCaLpTrI+8KSvW5MZkNEaClaMrtd6Ml9HpWSO/iy/P7fAf4sIvnABuAaLPF6QkSuBTYDX8iifUoEwbRNuPO0a7+Rcnls2MJFrhOKyNBTlJnZ4VVPWvGkZIqsCoUxZilQHWXX3O62RUkMfxon3PnDKpZSXQo1GJbMdp1QeKXTZLYVekp9OVlFSRS9B1GSImjSNeGu/RyIZCuWfGE5CjcnswNhCf+oyewuLCerKImiQqEkRSqdXqPhtUNPqc6BCO+F5FaPwmMLQaw26uHrUWgyW8kkKhRKUqTrztVrx9dTXZ3O266Fh73NZQOlzxYKJ08RdT2KUAuP9Hh6ihINFQolKaLFylPBCZsEU+xRFG1mttuEwmr6F7vhX/tktq5FoWQOFQolKdLmUdhzBFINPbXr9eTS0JPX7rDbWegpJBRBE+qoqyjpRj9aSlJkyqNIfuGitqVC3ZrMtpaLDVtrIjKZHV71ZLTqSckcKhRKUqRrQHLWUmibcJf86yGiPDYNZbu5hMeeUBfyKCL+Pm94MlurnpQMokKhJEwwaDCmbXW5ruCzB7lAKL+Q3DnDlwp161oM1oS7YEyPwtPFVu2KkigqFErCpHr3Hw2vR/AHUh/kPWGhp4BLk9lOh13nuneccIdOuFO6BRUKJWHaGvh1/WPjzCpOtSlgeDLbrS22nWS2k4OJVh4b3sJDdULJFNnu9aT0INoG5K6fy0lmpywUoXkU4Bd3JrM9djI7lIOJEnoKhqqe3CeUSu6gQqEkjD+NHoVTHptqOKt9eay9zWXJbF+8ZHZE1ZPOo1AyhQqFkjBty212/VwdPYrklCJUHhtoGyDd5lHES2a3W7goTe3fFSUamqNQEiY0qKch9uTE11Nd69kZEwPGvclsj8cKKcVKZotIaB6KVj0pmUSFQkmYVAf1aHgjwirJRrNEJOwcbed0Ez6Pp1133A7JbA/tVsDT0JOSKTT0pCRMrDvbVIhcCtWXQt7DacPtsW+rXaYToTU7gjGue2TVk9uEUskdVCiUhAnEuLNNBadFdlfmZng8zp20JTzisjtqJ5kdq3us89zYJcJuy9EouYMKhZIw6fQonBbawRiJ2kRw7qg9Iq4cJJ3gZQJ+AAAgAElEQVQ2J8EYIb/wVuvBIDqPQskYWc9RiIhXRD4Wkb/bz0eLyAciUiMij9vraSs5gJM0TotHIc5SqF0IPYUqp4KuDLt4PU4b9djdY8EScA09KZkk60IB3ACsDnv+C+BOY8w4oB64NitWKR1wksbpylF0JZntnMPp9eTGQdJJZgdjhZ5CjRG16knJLFkVChGpAs4DHrCfC3A68JR9yEPARdmxTonEH0oapyn01MW1JEKT9lzqUYS6x8ZKZtvfXkdMtOpJyRTZ9ih+DdwE2PeqDAAajDF++3ktMCwbhikdCabRo3AW3YkVVkkEb9hA6kah8ArtwnMxPQrj3mug5AZZEwoROR/YbYxZkuLr54vIYhFZXFdXl2brlGikc8lRr6QvmR0IunMZUK/H0+4adfQo2tbkcOs1UHKDbHoUJwIXisgm4DGskNNvgFIRcaqxqoBt0V5sjFlgjKk2xlQPGjSoO+zt9aRzyVFr0R26lMz2hCWz0+Hl5Bpej3XN/THE1NOu6kmXQlUyR9Y+WsaYW4wxVcaYUcBlwGvGmCuA14FL7MOuBp7LkolKBM4M4XQlswFaA04lVWrnsHIU7kxmO5MSQ+WxnVQ9aVNAJZPk4j3IzcC/ikgNVs7iD1m2R7FxkqrpKI+NFIqUk9muLo9tP+Eu2sxssHJHQaNLoSqZIycm3Blj3gDesB9vAGZl0x4lOoEuJJ4jcQb2I/4uCIWz+JERdwqFtK8Mi9brCexW6zozW8kgOSEUSs8gEKP6JhWcQa3FEYpUktn2cqqCcV2fJ7BzMAETU6DDcxRa9aRkklwMPSk5Sjo9Ck86PAq7X5Q/GEwpGZ7rOHNN4iWzLY9Cq56UzKEehZIwsRbQSQVHbI6kIUcB4sr4vCcimd1hhTtPmEehVU9KBkn4oyUiJ4nINfbjQSIyOnNmKblIrCU5U8EZ2Fu74FE4/aJcWx4r7ZPZ0dbMBp1wp2SehIRCRH6KVY10i70pD/jfTBml5CahluDpWLhI2nsUKU24C7UqT0/eJNeIbHPSIZkdCj2hLTyUjJKoR/E54ELgIIAxZjvQP1NGKblJIEY9fyr4InIUqbbwCE02c+EY6fEIxkBrjPkroV5PQZ1HoWSWRIXiiDHGAAZARPpmziQlV0mnUHjSkaOw14x2azI75HX5o7d3lw4zs1UolMyQ6LfrCRG5H6u9xteAV4HfZ84sJRfxp1EonLvhI/4gHiGl1emsmctBq+LHfTqB19u51+UNr3oyWvWkZI6Eqp6MMb8Skc8CTcBE4CfGmFcyapmSc8RqJZEKXntkbw2kPqva4xECBggGKfK5r4DPEYLWGHkcrXpSuou43y4R8QKvGmNOA1QcejFp9SjCwiqp3gk7a0rjcWd5rDciPBdrhbugsQoN3HgNlNwg7j2IMSYABEWkpBvsUXKYYDqrnpzQUyD10tbw8lg3JrPjtTlpF3rSFh5KBknUXz8ArBCRV7ArnwCMMddnxColJ2nrHtv1GIfjRbT6U78T9nqccJiEQlluwhGGlhh5HOeyaQsPJdMkKhTP2D9KL6atnr/r53Im7bV0IUfhrClNEFfG58M77Ea7Rp6wHIUxqRUEKEoiJJrMfkhE8oEJ9qY1xpjWzJml5CJdWWQoEk9YjiLl0JOdozBiXF8eGy2P06FVuwqFkiESEgoRORV4CNgECDBcRK42xryZOdOUXKOte2zXzxU+yKWazHbWlPa6NJkd3jgxmpg6180JCbrRq1Jyg0RDT7cDZxpj1gCIyATgUWBmpgxTco9AGj2K8ERtl8pjgwaDO2dm+8LFNMo1cq6bs5a5G8VSyQ0S/cbnOSIBYIxZi9XvSelFtHWP7fq5wsMqqecoJNRi283J7FiVYc6mI4H0VaMpSjQS9SgWi8gDtDUCvAJYnBmTlFwlYId50pE0DQ899clLbbKc0+vJiDvDLk5oqSWGmLZVjqXeBkVREiHRr9c3gVXA9fbPKntbyojIcBF5XURWicgnInKDvb1cRF4RkXX277KuvI+SPgImfbX67UJPKZ7TI5ZQWHkK9ymFL054LjKZrS08lEyR6LfLB/zGGHOxMeZi4C7A28X39gPfN8ZMAWYD14nIFOCHwEJjzHhgof1cyQECaWw8F5oj0KXy2LY23K70KOKIaYeqJ/UolAyR6NdrIVAY9rwQqzFgyhhjdhhjPrIf7wdWA8OAeVgVVti/L+rK+yjpI51C4UlDjsJJZvsDqXsluUz4mh3REtWh0FMgfbkjRYlGokJRYIw54DyxHxelywgRGQVMBz4AKowxO+xdO4GKdL2P0jXSKRThq+Slek6vHXoKGpcms8O6x3aWzNaqJyXTJPrtOigiM5wnIlINHE6HASLSD3ga+K4xpil8X/gaGFFeN19EFovI4rq6unSYosQhraEnSYNQ2B6FWzuntptw12mOQquelMySaLnJd4EnRWS7/XwocGlX31xE8rBE4s/GGKdFyC4RGWqM2SEiQ4Hd0V5rjFkALACorq6OKiZKevGnM/QUdp6UJ9zZ5bHGrR5FWA7C5+n4VQ0P34F6FErm6PTbJSLHicgQY8wiYBLwONAKvARs7Mobi1Vj+QdgtTHmjrBdzwNX24+vBp7ryvso6SMYNCm324gk/DypnjPkUbg0mR1eGaYtPJRsEu/rdT9wxH58AvAj4F6gHvtuvgucCFwJnC4iS+2fc4H/Bj4rIuuAM+znSg7gD6ZvXebw86R6J+wRsdZicGmL7fDKMF+UqedtM7PTt06IokQjXujJa4zZZz++FFhgjHkaeFpElnbljY0xb2P1jYrG3K6cW8kMQWOiDlipED6opTrIh3sibgw9tasMi3KNnE0aelIyTbxvl1dEHDGZC7wWts99a08qneJP4517u9BTiuLjaScUXTYp5wi/RlGT2R2WSu0eu5TeR7zB/lHgnyKyB6vK6S0AERkHNGbYNiXHCARTn/MQSbqS2W2P3acU3jh5HM1RKN1Fp0JhjPmZiCzEqnJ62S5XBcsT+U6mjVNyi5wrjxV3exTeOGLqiK3TZlxDT0qmiBs+Msa8H2Xb2syYo+QyaRWKdEy4S4NXkst444TnQqGnoM6jUDKLC+/DlEyRMY8iDaGndJXt5hLtKsOieRTaPVbpJlQolIRJ54S79vmFdCSz3TdIxptr4qRlQslsF14DJTdQoVASJpiBNuORj5PB7eWx8a5ReNPA8OeKkm7c9+1SMoY/oMns7iReZVjH9Si6xy6l9+HCr5eSKax1H9JfHpuO0JMbk9nx5pqICCJa9aRkHhUKJWHSmaOANoFIPZnd9jhdM8ZziXjJbGd7q7bwUDKMCoWSMMEMCUWqd8LheQk3ehSJVHV5RUJVT268BkpuoEKhJIw/jd1joc2TSLl7rIQPpO77KLebJxLjGnk8uhSqknnc9+1SMkYgjd1jISz0lLJHEf2xW0jYo9CqJyXDuPDrpWSKQDB93WMhLPSU4gCXSAy/J5NIZZjHI21rZuu3WckQ+tFSEiZgMuNRpCo+4a9zYzI7kRYlnnCPQkNPSoZQoVASJpDmHIUz+KlHEZ2EQk8eCZtH4b5roOQGKhRKwgSCJq21+r5QjiK117cfSN33UQ6/1DGT2SJt8yhUKJQM4b5vl5Ix0u1RtCWzU/sYtq8KSotJOYWItIXnYnoU0BrU0JOSWXL26yUiZ4vIGhGpEZEfZtseJb3dY6FtcE95wl0aOtDmOs7fFcujsKqetM24kllyUihExAvcC5wDTAEuF5Ep2bVKSbdQOOGiVBPR8dZrcAPxPAoRIRDUqicls+TqR2sWUGOM2WCMOQI8BszLsk29nkAau8dCWww+5WS2y3s9Qfw2J+nowqso8Yi7wl2WGAZsDXteCxyfqTfzB4I8t3Q7Xo9QUphHSVEeA/v2YcSAonbHLd3awO0vr2H97gMAzB4zgJ9ffDQFed6E3ysQNDzy3ibeWFvH8tpGfnbRUZxz9NAu2b9qexNfe3gx9185k6OGlbR7rzU79zN6YF8K86PbeMcra1m9o4n/d94URgwoYldTM4s31bOrqZl+BT6+UD287XwBk9Z23t4uJrN9Lk9mQ5iYdlL1FHrsUrFUsk+uCkVcRGQ+MB9gxIgRXTrXCyt28P0nl3XY/p3Tx/H9MyfiDwS59dmVPL54KwP75fOZCYNp8Qd4duk2Nu09yANXH0d53/yE3uu3r9dw+ytrGTuoLwCPvL+5y0LxTs0etjUc5sYnl/Hct0/EGLjvn+t5YtFWtjc2M6GiH/d9aSZjBvVr97otew9x7+s1BIKGt9bVcUxVKYs27SO0Mjpw3KhyRg+0bA0Yk9YZ0I7opCo+7cpj3akT+OwLHiv0FL5ZVCiUDJGrQrENGB72vMreFsIYswBYAFBdXW3oAi8s38GQ4gL+/LXjaTzcSuPhVp79aBt3v1bD9BGlLFy9m8cXb2X+KWO4fu54+vWxLtv5K3dw/WNLueDut/nPzx3FaRMHd/o+H27cx52vruWiaZXceek07nx1HXe/to7dTc0MLi5I2f5VO5rI93n4dOd+/u35VazY1sDKbU2cMmEQXzlpNL99Yz0X3vMORw0rpm5/C+cdU8n3zhjPva/X4PUIT39zDve9sZ6augN857RxfHbKEILGMO/ed3h7XV1IKKzusen0KOzfKY5vvSHs4omXzO4F10DJPrkqFIuA8SIyGksgLgO+mIk32t/cyhtr6/jS8SMZG3bHfcKYAazbfYBvPPIRRwJBvnnqWG4+e1K715591FAem1/AD55cxjV/XMSl1cP5788fHfXObuW2Rm547GNGlBfxn5+zjrnw2EruWriOvy3fwbUnjY5r60db6lm8aR/zTxnbbvvqHU2cNG4g5X3zefTDLfQv8PHAVdWcMaUCgHOPHspPnltJw6FWBvTtw10L17HnQAtPf1TLl2aPZNrwUu67cma7cxpjGF5eyJvr9nDlCaMAp3tsQpc1IZxQSTpWuHPjmtnQJqaxPQoNPSmZJyeFwhjjF5FvA/8HeIEHjTGfZOK9Xl29iyP+IOcd0z78U5Dn5bdXzOBzv32HCyZVctNZE6O+fsaIMl684WR+8Y81PPjORs46qoLTJ1WE9je3BrjpqeU8v2w7ZUV5/P6q6pBHMm5wP44aVszzS7clJBSPvLeZ55Zu45oTR5NnjyDNrQFqdh9g7uTBfP0zYxlaUsAlM6sYOaBv6HWVpYU8cPVxgCUAP3x6BX/5YAv5Xg/f+MzYqO8lIpw0bhB/X7ad1kCQPK8nAx5F1+ZR9IZktpN76ayFR+ixS8NvSvbJSaEAMMa8CLyY6fd5YfkOKksKmD68tMO+0QP78v4tc+Mmq/v4vNxy7iReX7Obn7/4KaeMHxSKLb+8ahfPL9vO104ezXfmjqe4IK/day88tpKfv/gpNbv3M7y8iD6+2O9Vs/sAQQO7mpqpKisKbfMHDVOGllBckMf3z4wuaA4iws8vPpriQh9DSwoZUhI75HXK+IE8+uEWlm1tYMaIMiC9d+5dTWa7vc04tA3+nbXwiPZYUdKJO79dCdJ4uJV/rq3j3KOHxowBJ1rRlOf1cPPZk6jZfYDHF7cVbO1oOAzADWdM6CASABccW4kInHHHm0z88Uvc89q6qOcPBg3r66xqq+0NzaHtq3Y0ATClsjghO8EaUG49bwpfiePFzBk7EI/Am+v24M/AKmpd7fXk9pnZED881xu8KiX7uPTrlRivrNpFa8Bw/rGVaTnfWVMrOG5UGb9+dR1Be2Dd0dhMvz6+ULgpkqElhdxz+Qx+cNZEKksKWLK5Pupx2xsPc+hIwHpsiw9YpbFF+V5GlhdFfV1XKCnK45iqUt5eV0fQpF8onEly6Zhw59a76XirAIZfOrdeAyX75GzoqTs47+ih9C/wcWxVSfyDE0BEmDdtGD/+60p2729hSEkBu5qaqSju07kddn5keW0DG+oORj2mxp67AbAtXCh2NDFpSP+0NusL55TxA7nn9RrqDx0B0pswTadH4dZBMt7MbE1mK91Br/YoCvO9nDV1SFrrz4eVFQKwreEQADubmjvNA7R7bWkR2xoOY0zHal9HKPr4PCGhMMawekcTk4cmHnZKlmOqSgkaWLfLev9MrJmd6jl7wyAZT0zDbxBcegmUHKBXC0UmqCq1hKK23hrMdzU2U5HgHImqskIOHQlQf6i1w76a3Qco75vP+Ip+odBTbf1h9jf7k8pPJEtVufX3bN5reTrp7fXU+d1yoq+3HrvzoxwvPBdqGig64U7JHO78dmURx6OorT9MIGjYvb+FoYl6FI43Un+4w76a3QcYN6gflSWFof1OIjuTHsWwUkcoLA8pl5LZnl6UzI51jbrqlSlKIrj065U9ivJ9lPfNZ1vDYfYeaMEfNAxJ0KNwBmUnbOVgjGHd7gOMq+jHsLJCttvhqeW1DXg9wuQhmROK/gV5lBblsSkDQtHVpVB7Q47CE/K6on9Vnf1a8aRkkl6dzM4Uw0qtu/6dTVYZa6Khp+H23AgnbPX0klrqDx1h3rRhNB5uZdygfgSN4eCRAE2H/Szd2sCkIf1jNvxLF1VlhWzZl/7Qk7eLg1y79ShcKhTxVgF0/my3/v1KbqBCkQGGlRZSU3eAnY2WUCSazC4utMpoHaG49/UaNuw5yN6DVsXRuMH9ONDiB2Br/SGWb23kwmnpKe3tjKrSIl5fsxtIb9K4q2GT3tA5NV54LjTPwqV/v5IbaOgpA1SVFVJbf4hdtkeRaOhJROzXHqbxUCsb9hxEBH73xnrAEgonPPXmujr2t/iZFmVGebqpKiukxW8tt5nOBYK62uupN4Se4oXnQqEnl/79Sm6gQpEBhpUV0twaZNWOJnweYUC/zudRtHttaSHbGg6ztLYBgFvPnUy+10PffC9DSwqotIXixRU7AJg+onuEwiGdsXBncEv1brht4SP3VvzEC8+FVz0pSqbQ0FMGcO76F2+qZ3D/Pknd7VaVFfLhpn18vKUeEbj0uOGU981ne8NhRIQBffPJ93lYua2J/gU+xgzsF/+kXcTpKwXp7fXk62LoSUTwesTVg6Q3TjJbq56U7kCFIgM4A+u63QeSvuMfVlbI/mY/b63bw/jB/ehfkMfFM6pC+z0eYVhpIRv3HOTYqtJuCTk4cymc908XnjQMcl4R15bGQpuYxvobJeRVqVAomcPFX7HsMSwsVJNofsLBEZklm+tj5h8qS61zdkd+Ato8JEhz99gu5ijAGkDdnMh1BEA9CiWbqFBkgJLCPPrbTQATrXhyCB+Up9utvSOpLLGO6S6h6F+QR0mh1fk2nR5FV8tjwRIJNw+S8Vqxx5uQpyjpQIUiQzheRfIeRZtQxBKCkQOKEIFp3ZDIdnDsysR6FF2ppPJ6eodQxOv15OZroGQfzVFkiKqyQj7duT9pj6K8bz4FeR48Ikyo6B/1mCtnj2LGyDIGJlFN1VWqygr5ZHtTZuZRdMWj6CVCETP0lIbwnaLEQ4UiQzghpERnZTuICCPL+1LeNz/ml7+kKI85Ywd22cZkcHInGZmZ3ZVkttuFwgktxZqZbW938SVQcoCsCIWI/BK4ADgCrAeuMcY02PtuAa4FAsD1xpj/y4aNXcUZWJMNPQHceem0jLflSBYn9JRWoQglarsoFC6Oz8fzKLraWFFREiFbHsUrwC3GGL+I/AK4BbhZRKYAlwFTgUrgVRGZYIwJZMnOlJk33WqtMXJA8ivPZbJteKqMGWTN1+hXkL6PTDoa2nlF8KZxtniuETeZrTkKpRvISjLbGPOyMcZvP30fcCYKzAMeM8a0GGM2AjXArGzY2FUG9y/ga6eMcc2M4VPGD+Tpb57ApDR2qvWlIZntcblH0ZasVo9CyR65UPX0FeAf9uNhwNawfbX2NiXLiAgzR5an9ZyazI6PL841Uo9C6Q4yFnoSkVeBIVF23WqMec4+5lbAD/w5hfPPB+YDjBgxoguWKtkidDesyeyYeOIls52Z2S6+Bkr2yZhQGGPO6Gy/iHwZOB+Ya9oWid4GDA87rMreFu38C4AFANXV1R0XmVZynq4uhQp2Cw8Xh13iJrNDHke3maT0QrISehKRs4GbgAuNMeHLuT0PXCYifURkNDAe+DAbNiqZZ+KQ/kweWkzfPqnfr3g9ktbW57lGvF5POo9C6Q6yVfV0D9AHeMVO9r5vjPmGMeYTEXkCWIUVkrquJ1Y8KYlxyoRBnDJhUJfO4ZHekcyO1+vJLUUTSm6SFaEwxozrZN/PgJ91ozlKD8bndXeOIt4Kdp44+xUlHeRC1ZOipIyntzQFjLXCnYaelG5AhULp0VgLF7l3kIxXQuxMxNOqJyWTaK8npUfzuenDXH03PbSkgAF988mLs2a2i/P5Sg6gQqH0aL40e2S2Tcgon59RxfnHVOKL0cNDq56U7kBDT4qSw3g80mmDyHQs/qQo8VChUJQejGivJ6UbUKFQlB6Mk5vQ0JOSSVQoFKUHk47FnxQlHioUitKD0aonpTtQoVCUHow3DR14FSUeKhSK0oPRFh5Kd6BCoSg9mLYV8FQolMyhQqEoPRht4aF0ByoUitKDaVszO8uGKK5GhUJRejDpWHdcUeKhQqEoPZh0rDuuKPFQoVCUHoxWPSndgQqFovRgvFr1pHQDWRUKEfm+iBgRGWg/FxG5S0RqRGS5iMzIpn2Kkuto1ZPSHWRNKERkOHAmsCVs8znAePtnPvC7LJimKD0GDT0p3UE2PYo7gZsAE7ZtHvCwsXgfKBWRoVmxTlF6AFoeq3QHWREKEZkHbDPGLIvYNQzYGva81t6mKEoUtHus0h1kbClUEXkVGBJl163Aj7DCTl05/3ys8BQjRozoyqkUpceioSelO8iYUBhjzoi2XUSOBkYDy+zVuaqAj0RkFrANGB52eJW9Ldr5FwALAKqrq020YxTF7ahHoXQH3R56MsasMMYMNsaMMsaMwgovzTDG7ASeB66yq59mA43GmB3dbaOi9BScqictj1UyScY8ihR5ETgXqAEOAddk1xxFyW1EQ09KN5B1obC9CuexAa7LnjWK0rPQhYuU7kBnZitKD8arS6Eq3YAKhaL0YLQpoNIdqFAoSg8mVPWkOQolg6hQKEoPxnEktOpJySQqFIrSg/HowkVKN6BCoSg9GJ9OuFO6ARUKRenBDC8r4rrTxvKZCYOybYriYrI+j0JRlNTxeIQfnDUp22YoLkc9CkVRFKVTVCgURVGUTlGhUBRFUTpFhUJRFEXpFBUKRVEUpVNUKBRFUZROUaFQFEVROkWFQlEURekUsdYK6tmISB2wOcmXDQT2ZMCcdKN2po+eYCOonemkJ9gI2bNzpDEm7rR+VwhFKojIYmNMdbbtiIfamT56go2gdqaTnmAj5L6dGnpSFEVROkWFQlEURemU3iwUC7JtQIKonemjJ9gIamc66Qk2Qo7b2WtzFIqiKEpi9GaPQlEURUmAXikUInK2iKwRkRoR+WG27QEQkeEi8rqIrBKRT0TkBnt7uYi8IiLr7N9l2bYVQES8IvKxiPzdfj5aRD6wr+njIpKfAzaWishTIvKpiKwWkRNy7XqKyPfs//dKEXlURApy4VqKyIMisltEVoZti3rtxOIu297lIjIjy3b+0v6fLxeRZ0WkNGzfLbada0TkrGzaGbbv+yJiRGSg/Txr1zMWvU4oRMQL3AucA0wBLheRKdm1CgA/8H1jzBRgNnCdbdcPgYXGmPHAQvt5LnADsDrs+S+AO40x44B64NqsWNWe3wAvGWMmAcdi2Zsz11NEhgHXA9XGmKMAL3AZuXEt/wScHbEt1rU7Bxhv/8wHftdNNkJ0O18BjjLGHAOsBW4BsL9PlwFT7df81h4PsmUnIjIcOBPYErY5m9czKr1OKIBZQI0xZoMx5gjwGDAvyzZhjNlhjPnIfrwfa1AbhmXbQ/ZhDwEXZcfCNkSkCjgPeMB+LsDpwFP2IVm3U0RKgFOAPwAYY44YYxrIvevpAwpFxAcUATvIgWtpjHkT2BexOda1mwc8bCzeB0pFZGi27DTGvGyM8dtP3weqwux8zBjTYozZCNRgjQdZsdPmTuAmIDxZnLXrGYveKBTDgK1hz2vtbTmDiIwCpgMfABXGmB32rp1ARZbMCufXWB/uoP18ANAQ9uXMhWs6GqgD/miHyB4Qkb7k0PU0xmwDfoV1N7kDaASWkHvX0iHWtcvl79RXgH/Yj3PKThGZB2wzxiyL2JVTdkLvFIqcRkT6AU8D3zXGNIXvM1aJWlbL1ETkfGC3MWZJNu1IAB8wA/idMWY6cJCIMFO2r6cd45+HJWqVQF+ihCdykWxfu0QQkVuxQrp/zrYtkYhIEfAj4CfZtiUReqNQbAOGhz2vsrdlHRHJwxKJPxtjnrE373LcTvv37mzZZ3MicKGIbMIK252OlQsotcMnkBvXtBaoNcZ8YD9/Cks4cul6ngFsNMbUGWNagWewrm+uXUuHWNcu575TIvJl4HzgCtM2ByCX7ByLdYOwzP4uVQEficgQcstOoHcKxSJgvF1Zko+V3Ho+yzY5cf4/AKuNMXeE7XoeuNp+fDXwXHfbFo4x5hZjTJUxZhTWtXvNGHMF8DpwiX1YLti5E9gqIhPtTXOBVeTW9dwCzBaRIvv/79iYU9cyjFjX7nngKrtaZzbQGBai6nZE5Gys0OiFxphDYbueBy4TkT4iMhorWfxhNmw0xqwwxgw2xoyyv0u1wAz7c5tT1xMAY0yv+wHOxaqGWA/cmm17bJtOwnLllwNL7Z9zseL/C4F1wKtAebZtDbP5VODv9uMxWF+6GuBJoE8O2DcNWGxf078CZbl2PYHbgE+BlcAjQJ9cuJbAo1h5k1asQezaWNcOEKxKwvXACqwqrmzaWYMV43e+R/eFHX+rbeca4Jxs2hmxfxMwMNvXM9aPzsxWFEVROqU3hp4URVGUJFChUBRFUTpFhUJRFEXpFBUKRVEUpVNUKBRFUZROUaFQejUiEhCRpWE/nTktMfQAAAJzSURBVDYJFJFviMhVaXjfTU630CRfd5aI3GZ3cv1H/FcoStfxxT9EUVzNYWPMtEQPNsbcl0ljEuBkrAl5JwNvZ9kWpZegHoWiRMG+4/8fEVkhIh+KyDh7+7+JyI324+vFWj9kuYg8Zm8rF5G/2tveF5Fj7O0DRORlsdaeeABrUpXzXl+y32OpiNwfrfW1iFwqIkux2pL/Gvg9cI2IZL2rgOJ+VCiU3k5hROjp0rB9jcaYo4F7sAbnSH4ITDfWugffsLfdBnxsb/sR8LC9/afA28aYqcCzwAgAEZkMXAqcaHs2AeCKyDcyxjyO1VF4pW3TCvu9L+zKH68oiaChJ6W301no6dGw33dG2b8c+LOI/BWrRQhYrVg+D2CMec32JIqx1sa42N7+gojU28fPBWYCi6x2TxQSu1HhBGCD/bivsdYtUZSMo0KhKLExMR47nIclABcAt4rI0Sm8hwAPGWNu6fQgkcXAQMAnIquAoXYo6jvGmLdSeF9FSRgNPSlKbC4N+/1e+A4R8QDDjTGvAzcDJUA/4C3s0JGInArsMda6Im8CX7S3n4PVoBCsJnuXiMhge1+5iIyMNMQYUw28gLV+xf9gNbOcpiKhdAfqUSi9nUL7ztzhJWOMUyJbJiLLgRbg8ojXeYH/tZdcFeAuY0yDiPwb8KD9ukO0teW+DXhURD4B3sVeI9kYs0pEfgy8bItPK3AdsDmKrTOwktnfAu6Isl9RMoJ2j1WUKNiLyVQbY/Zk2xZFyTYaelIURVE6RT0KRVEUpVPUo1AURVE6RYVCURRF6RQVCkVRFKVTVCgURVGUTlGhUBRFUTpFhUJRFEXplP8PUT7UsLc4aK4AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "def cem(n_iterations=500, max_t=1000, gamma=1.0, print_every=10, pop_size=50, elite_frac=0.2, sigma=0.5):\n",
    "    \"\"\"PyTorch implementation of the cross-entropy method.\n",
    "        \n",
    "    Params\n",
    "    ======\n",
    "        n_iterations (int): maximum number of training iterations\n",
    "        max_t (int): maximum number of timesteps per episode\n",
    "        gamma (float): discount rate\n",
    "        print_every (int): how often to print average score (over last 100 episodes)\n",
    "        pop_size (int): size of population at each iteration\n",
    "        elite_frac (float): percentage of top performers to use in update\n",
    "        sigma (float): standard deviation of additive noise\n",
    "    \"\"\"\n",
    "    n_elite=int(pop_size*elite_frac)\n",
    "\n",
    "    scores_deque = deque(maxlen=100)\n",
    "    scores = []\n",
    "    best_weight = sigma*np.random.randn(agent.get_weights_dim())\n",
    "\n",
    "    for i_iteration in range(1, n_iterations+1):\n",
    "        weights_pop = [best_weight + (sigma*np.random.randn(agent.get_weights_dim())) for i in range(pop_size)]\n",
    "        rewards = np.array([agent.evaluate(weights, gamma, max_t) for weights in weights_pop])\n",
    "\n",
    "        elite_idxs = rewards.argsort()[-n_elite:]\n",
    "        elite_weights = [weights_pop[i] for i in elite_idxs]\n",
    "        best_weight = np.array(elite_weights).mean(axis=0)\n",
    "\n",
    "        reward = agent.evaluate(best_weight, gamma=1.0)\n",
    "        scores_deque.append(reward)\n",
    "        scores.append(reward)\n",
    "        \n",
    "        torch.save(agent.state_dict(), 'checkpoint.pth')\n",
    "        \n",
    "        if i_iteration % print_every == 0:\n",
    "            print('Episode {}\\tAverage Score: {:.2f}'.format(i_iteration, np.mean(scores_deque)))\n",
    "\n",
    "        if np.mean(scores_deque)>=90.0:\n",
    "            print('\\nEnvironment solved in {:d} iterations!\\tAverage Score: {:.2f}'.format(i_iteration-100, np.mean(scores_deque)))\n",
    "            break\n",
    "    return scores\n",
    "\n",
    "scores = cem()\n",
    "\n",
    "# plot the scores\n",
    "fig = plt.figure()\n",
    "ax = fig.add_subplot(111)\n",
    "plt.plot(np.arange(1, len(scores)+1), scores)\n",
    "plt.ylabel('Score')\n",
    "plt.xlabel('Episode #')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 4. Watch a Smart Agent!\n",
    "\n",
    "In the next code cell, you will load the trained weights from file to watch a smart agent!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# load the weights from file\n",
    "agent.load_state_dict(torch.load('checkpoint.pth'))\n",
    "\n",
    "state = env.reset()\n",
    "while True:\n",
    "    state = torch.from_numpy(state).float().to(device)\n",
    "    with torch.no_grad():\n",
    "        action = agent(state)\n",
    "    env.render()\n",
    "    next_state, reward, done, _ = env.step(action)\n",
    "    state = next_state\n",
    "    if done:\n",
    "        break\n",
    "\n",
    "env.close()"
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
